package sparktest;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;

/**
 * Created by root on 2018/11/29.
 */
public class WordCountLambda {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("WordCount")
                .setMaster("local[*]");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        JavaRDD<String> lines = jsc.textFile("hdfs://bigdata01:9000/word.txt");

        JavaRDD<String> words = lines.flatMap(line -> Arrays.asList(line.split(" ")).iterator());

        JavaPairRDD<String, Integer> wordAndOne = words.mapToPair(word -> new Tuple2<String, Integer>(word, 1));

        JavaPairRDD<String, Integer> wordCount = wordAndOne.reduceByKey((a, b) -> a + b);

        JavaPairRDD<Integer, String> countWord = wordCount.mapToPair(t -> t.swap());

        JavaPairRDD<Integer, String> countWordSorted = countWord.sortByKey();

        JavaPairRDD<String, Integer> wordCountSorted = countWordSorted.mapToPair(t -> t.swap());

        List<Tuple2<String, Integer>> collect = wordCountSorted.collect();

        System.out.println(collect);

    }
}
